{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "titanic_survival = pd.read_csv(\"titanic_train.csv\")\n",
    "titanic_survival.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177\n"
     ]
    }
   ],
   "source": [
    "#The Pandas library uses NaN, which stands for \"not a number\", to indicate a missing value.\n",
    "#we can use the pandas.isnull() function which takes a pandas series and returns a series of True and False values\n",
    "age = titanic_survival[\"Age\"]\n",
    "#print(age.loc[0:10])\n",
    "age_is_null = pd.isnull(age)\n",
    "#print (age_is_null)\n",
    "age_null_true = age[age_is_null]\n",
    "#print (age_null_true)\n",
    "age_null_count = len(age_null_true)\n",
    "print(age_null_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan\n"
     ]
    }
   ],
   "source": [
    "#The result of this is that mean_age would be nan. This is because any calculations we do with a null value also result in a null value\n",
    "mean_age = sum(titanic_survival[\"Age\"]) / len(titanic_survival[\"Age\"])\n",
    "print (mean_age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29.6991176471\n"
     ]
    }
   ],
   "source": [
    "#we have to filter out the missing values before we calculate the mean.\n",
    "good_ages = titanic_survival[\"Age\"][age_is_null == False]\n",
    "#print good_ages\n",
    "correct_mean_age = sum(good_ages) / len(good_ages)\n",
    "print (correct_mean_age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29.69911764705882\n"
     ]
    }
   ],
   "source": [
    "# missing data is so common that many pandas methods automatically filter for it\n",
    "correct_mean_age = titanic_survival[\"Age\"].mean()\n",
    "print (correct_mean_age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 84.15468749999992, 2: 20.66218315217391, 3: 13.675550101832997}\n"
     ]
    }
   ],
   "source": [
    "#mean fare for each class\n",
    "passenger_classes = [1, 2, 3]\n",
    "fares_by_class = {}\n",
    "for this_class in passenger_classes:\n",
    "    pclass_rows = titanic_survival[titanic_survival[\"Pclass\"] == this_class]\n",
    "    pclass_fares = pclass_rows[\"Fare\"]\n",
    "    fare_for_class = pclass_fares.mean()\n",
    "    fares_by_class[this_class] = fare_for_class\n",
    "print (fares_by_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pclass\n",
      "1    0.629630\n",
      "2    0.472826\n",
      "3    0.242363\n",
      "Name: Survived, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#index tells the method which column to group by\n",
    "#values is the column that we want to apply the calculation to\n",
    "#aggfunc specifies the calculation we want to perform\n",
    "passenger_survival = titanic_survival.pivot_table(index=\"Pclass\", values=\"Survived\", aggfunc=np.mean)\n",
    "print (passenger_survival)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pclass\n",
      "1    38.233441\n",
      "2    29.877630\n",
      "3    25.140620\n",
      "Name: Age, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "passenger_age = titanic_survival.pivot_table(index=\"Pclass\", values=\"Age\")\n",
    "print(passenger_age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                Fare  Survived\n",
      "Embarked                      \n",
      "C         10072.2962        93\n",
      "Q          1022.2543        30\n",
      "S         17439.3988       217\n"
     ]
    }
   ],
   "source": [
    "port_stats = titanic_survival.pivot_table(index=\"Embarked\", values=[\"Fare\",\"Survived\"], aggfunc=np.sum)\n",
    "print(port_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#specifying axis=1 or axis='columns' will drop any columns that have null values\n",
    "drop_na_columns = titanic_survival.dropna(axis=1)\n",
    "new_titanic_survival = titanic_survival.dropna(axis=0,subset=[\"Age\", \"Sex\"])\n",
    "#print new_titanic_survival"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28.0\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "row_index_83_age = titanic_survival.loc[83,\"Age\"]\n",
    "row_index_1000_pclass = titanic_survival.loc[766,\"Pclass\"]\n",
    "print (row_index_83_age)\n",
    "print (row_index_1000_pclass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     PassengerId  Survived  Pclass                                  Name  \\\n",
      "630          631         1       1  Barkworth, Mr. Algernon Henry Wilson   \n",
      "851          852         0       3                   Svensson, Mr. Johan   \n",
      "493          494         0       1               Artagaveytia, Mr. Ramon   \n",
      "96            97         0       1             Goldschmidt, Mr. George B   \n",
      "116          117         0       3                  Connors, Mr. Patrick   \n",
      "672          673         0       2           Mitchell, Mr. Henry Michael   \n",
      "745          746         0       1          Crosby, Capt. Edward Gifford   \n",
      "33            34         0       2                 Wheadon, Mr. Edward H   \n",
      "54            55         0       1        Ostby, Mr. Engelhart Cornelius   \n",
      "280          281         0       3                      Duane, Mr. Frank   \n",
      "\n",
      "      Sex   Age  SibSp  Parch      Ticket     Fare Cabin Embarked  \n",
      "630  male  80.0      0      0       27042  30.0000   A23        S  \n",
      "851  male  74.0      0      0      347060   7.7750   NaN        S  \n",
      "493  male  71.0      0      0    PC 17609  49.5042   NaN        C  \n",
      "96   male  71.0      0      0    PC 17754  34.6542    A5        C  \n",
      "116  male  70.5      0      0      370369   7.7500   NaN        Q  \n",
      "672  male  70.0      0      0  C.A. 24580  10.5000   NaN        S  \n",
      "745  male  70.0      1      1   WE/P 5735  71.0000   B22        S  \n",
      "33   male  66.0      0      0  C.A. 24579  10.5000   NaN        S  \n",
      "54   male  65.0      0      1      113509  61.9792   B30        C  \n",
      "280  male  65.0      0      0      336439   7.7500   NaN        Q  \n",
      "-------------\n",
      "    PassengerId  Survived  Pclass                                  Name   Sex  \\\n",
      "0           631         1       1  Barkworth, Mr. Algernon Henry Wilson  male   \n",
      "1           852         0       3                   Svensson, Mr. Johan  male   \n",
      "2           494         0       1               Artagaveytia, Mr. Ramon  male   \n",
      "3            97         0       1             Goldschmidt, Mr. George B  male   \n",
      "4           117         0       3                  Connors, Mr. Patrick  male   \n",
      "5           673         0       2           Mitchell, Mr. Henry Michael  male   \n",
      "6           746         0       1          Crosby, Capt. Edward Gifford  male   \n",
      "7            34         0       2                 Wheadon, Mr. Edward H  male   \n",
      "8            55         0       1        Ostby, Mr. Engelhart Cornelius  male   \n",
      "9           281         0       3                      Duane, Mr. Frank  male   \n",
      "10          457         0       1             Millet, Mr. Francis Davis  male   \n",
      "\n",
      "     Age  SibSp  Parch      Ticket     Fare Cabin Embarked  \n",
      "0   80.0      0      0       27042  30.0000   A23        S  \n",
      "1   74.0      0      0      347060   7.7750   NaN        S  \n",
      "2   71.0      0      0    PC 17609  49.5042   NaN        C  \n",
      "3   71.0      0      0    PC 17754  34.6542    A5        C  \n",
      "4   70.5      0      0      370369   7.7500   NaN        Q  \n",
      "5   70.0      0      0  C.A. 24580  10.5000   NaN        S  \n",
      "6   70.0      1      1   WE/P 5735  71.0000   B22        S  \n",
      "7   66.0      0      0  C.A. 24579  10.5000   NaN        S  \n",
      "8   65.0      0      1      113509  61.9792   B30        C  \n",
      "9   65.0      0      0      336439   7.7500   NaN        Q  \n",
      "10  65.0      0      0       13509  26.5500   E38        S  \n"
     ]
    }
   ],
   "source": [
    "new_titanic_survival = titanic_survival.sort_values(\"Age\",ascending=False)\n",
    "print (new_titanic_survival[0:10])\n",
    "titanic_reindexed = new_titanic_survival.reset_index(drop=True)\n",
    "print('-------------')\n",
    "print(titanic_reindexed.loc[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PassengerId                  100\n",
      "Survived                       0\n",
      "Pclass                         2\n",
      "Name           Kantor, Mr. Sinai\n",
      "Sex                         male\n",
      "Age                           34\n",
      "SibSp                          1\n",
      "Parch                          0\n",
      "Ticket                    244367\n",
      "Fare                          26\n",
      "Cabin                        NaN\n",
      "Embarked                       S\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# This function returns the hundredth item from a series\n",
    "def hundredth_row(column):\n",
    "    # Extract the hundredth item\n",
    "    hundredth_item = column.loc[99]\n",
    "    return hundredth_item\n",
    "\n",
    "# Return the hundredth item from each column\n",
    "hundredth_row = titanic_survival.apply(hundredth_row)\n",
    "print (hundredth_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PassengerId      0\n",
      "Survived         0\n",
      "Pclass           0\n",
      "Name             0\n",
      "Sex              0\n",
      "Age            177\n",
      "SibSp            0\n",
      "Parch            0\n",
      "Ticket           0\n",
      "Fare             0\n",
      "Cabin          687\n",
      "Embarked         2\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "def not_null_count(column):\n",
    "    column_null = pd.isnull(column)\n",
    "    null = column[column_null]\n",
    "    return len(null)\n",
    "\n",
    "column_null_count = titanic_survival.apply(not_null_count)\n",
    "print (column_null_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0       Third Class\n",
      "1       First Class\n",
      "2       Third Class\n",
      "3       First Class\n",
      "4       Third Class\n",
      "5       Third Class\n",
      "6       First Class\n",
      "7       Third Class\n",
      "8       Third Class\n",
      "9      Second Class\n",
      "10      Third Class\n",
      "11      First Class\n",
      "12      Third Class\n",
      "13      Third Class\n",
      "14      Third Class\n",
      "15     Second Class\n",
      "16      Third Class\n",
      "17     Second Class\n",
      "18      Third Class\n",
      "19      Third Class\n",
      "20     Second Class\n",
      "21     Second Class\n",
      "22      Third Class\n",
      "23      First Class\n",
      "24      Third Class\n",
      "25      Third Class\n",
      "26      Third Class\n",
      "27      First Class\n",
      "28      Third Class\n",
      "29      Third Class\n",
      "           ...     \n",
      "861    Second Class\n",
      "862     First Class\n",
      "863     Third Class\n",
      "864    Second Class\n",
      "865    Second Class\n",
      "866    Second Class\n",
      "867     First Class\n",
      "868     Third Class\n",
      "869     Third Class\n",
      "870     Third Class\n",
      "871     First Class\n",
      "872     First Class\n",
      "873     Third Class\n",
      "874    Second Class\n",
      "875     Third Class\n",
      "876     Third Class\n",
      "877     Third Class\n",
      "878     Third Class\n",
      "879     First Class\n",
      "880    Second Class\n",
      "881     Third Class\n",
      "882     Third Class\n",
      "883    Second Class\n",
      "884     Third Class\n",
      "885     Third Class\n",
      "886    Second Class\n",
      "887     First Class\n",
      "888     Third Class\n",
      "889     First Class\n",
      "890     Third Class\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "#By passing in the axis=1 argument, we can use the DataFrame.apply() method to iterate over rows instead of columns.\n",
    "def which_class(row):\n",
    "    pclass = row['Pclass']\n",
    "    if pd.isnull(pclass):\n",
    "        return \"Unknown\"\n",
    "    elif pclass == 1:\n",
    "        return \"First Class\"\n",
    "    elif pclass == 2:\n",
    "        return \"Second Class\"\n",
    "    elif pclass == 3:\n",
    "        return \"Third Class\"\n",
    "\n",
    "classes = titanic_survival.apply(which_class, axis=1)\n",
    "print (classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0        adult\n",
      "1        adult\n",
      "2        adult\n",
      "3        adult\n",
      "4        adult\n",
      "5      unknown\n",
      "6        adult\n",
      "7        minor\n",
      "8        adult\n",
      "9        minor\n",
      "10       minor\n",
      "11       adult\n",
      "12       adult\n",
      "13       adult\n",
      "14       minor\n",
      "15       adult\n",
      "16       minor\n",
      "17     unknown\n",
      "18       adult\n",
      "19     unknown\n",
      "20       adult\n",
      "21       adult\n",
      "22       minor\n",
      "23       adult\n",
      "24       minor\n",
      "25       adult\n",
      "26     unknown\n",
      "27       adult\n",
      "28     unknown\n",
      "29     unknown\n",
      "        ...   \n",
      "861      adult\n",
      "862      adult\n",
      "863    unknown\n",
      "864      adult\n",
      "865      adult\n",
      "866      adult\n",
      "867      adult\n",
      "868    unknown\n",
      "869      minor\n",
      "870      adult\n",
      "871      adult\n",
      "872      adult\n",
      "873      adult\n",
      "874      adult\n",
      "875      minor\n",
      "876      adult\n",
      "877      adult\n",
      "878    unknown\n",
      "879      adult\n",
      "880      adult\n",
      "881      adult\n",
      "882      adult\n",
      "883      adult\n",
      "884      adult\n",
      "885      adult\n",
      "886      adult\n",
      "887      adult\n",
      "888    unknown\n",
      "889      adult\n",
      "890      adult\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "def is_minor(row):\n",
    "    if row[\"Age\"] < 18:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "minors = titanic_survival.apply(is_minor, axis=1)\n",
    "#print minors\n",
    "\n",
    "def generate_age_label(row):\n",
    "    age = row[\"Age\"]\n",
    "    if pd.isnull(age):\n",
    "        return \"unknown\"\n",
    "    elif age < 18:\n",
    "        return \"minor\"\n",
    "    else:\n",
    "        return \"adult\"\n",
    "\n",
    "age_labels = titanic_survival.apply(generate_age_label, axis=1)\n",
    "print (age_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age_labels\n",
      "adult      0.381032\n",
      "minor      0.539823\n",
      "unknown    0.293785\n",
      "Name: Survived, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "titanic_survival['age_labels'] = age_labels\n",
    "age_group_survival = titanic_survival.pivot_table(index=\"age_labels\", values=\"Survived\")\n",
    "print (age_group_survival)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
